diff --git a/debian/control b/debian/control index 2b7550668..d526dc513 100644 --- a/debian/control +++ b/debian/control @@ -1,32 +1,33 @@ Source: swh-web-ui Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, libjs-cryptojs, libjs-jquery-flot, libjs-jquery-flot-tooltip, python3-all, python3-docutils, python3-flask-testing, python3-blinker, python3-nose, python3-setuptools, python3-swh.core (>= 0.0.20~), python3-swh.storage (>= 0.0.72~), python3-vcversioner, - python3-yaml + python3-yaml, + python3-pygments Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DWUI/ Package: python3-swh.web.ui Architecture: all Depends: libjs-cryptojs, libjs-jquery-flot, libjs-jquery-flot-tooltip, python3-swh.core (>= 0.0.20~), python3-swh.storage (>= 0.0.72~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage Web UI diff --git a/requirements.txt b/requirements.txt index c71228e62..9ab67157a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,20 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html # Runtime dependencies Flask swh.core >= 0.0.20 swh.storage >= 0.0.72 dateutil docutils +pygments # Test dependencies #Flask-Testing #blinker # Non-Python dependencies #libjs-cryptojs #libjs-jquery-flot #libjs-jquery-flot-tooltip diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py index d81fa06bb..8220c58ef 100644 --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -1,138 +1,138 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from flask import Flask from swh.core import config -from swh.web.ui.renderers import urlize_api_links -from swh.web.ui.renderers import safe_docstring_display -from swh.web.ui.renderers import revision_id_from_url +from swh.web.ui.renderers import urlize_api_links, safe_docstring_display +from swh.web.ui.renderers import revision_id_from_url, highlight_source from swh.web.ui.renderers import SWHMultiResponse from swh.storage import get_storage DEFAULT_CONFIG = { 'storage_args': ('list[str]', ['http://localhost:5000/']), 'storage_class': ('str', 'remote_storage'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 6543), 'secret_key': ('string', 'development key'), 'max_log_revs': ('int', 25), } # api's definition app = Flask(__name__) app.response_class = SWHMultiResponse app.jinja_env.filters['urlize_api_links'] = urlize_api_links app.jinja_env.filters['safe_docstring_display'] = safe_docstring_display app.jinja_env.filters['revision_id_from_url'] = revision_id_from_url +app.jinja_env.filters['highlight_source'] = highlight_source def read_config(config_file): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict""" conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir') conf['storage'] = get_storage(conf['storage_class'], conf['storage_args']) return conf def load_controllers(): """Load the controllers for the application. """ from swh.web.ui import views, apidoc # flake8: noqa def rules(): """Returns rules from the application in dictionary form. Beware, must be called after swh.web.ui.main.load_controllers funcall. Returns: Generator of application's rules. """ for rule in app.url_map._rules: yield {'rule': rule.rule, 'methods': rule.methods, 'endpoint': rule.endpoint} def storage(): """Return the current application's storage. """ return app.config['conf']['storage'] def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration. Note: This function is called on a per-request basis so beware the side effects here! """ load_controllers() config_path = '/etc/softwareheritage/webapp/webapp.ini' conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'), level=logging.INFO) return app(environ, start_response) def run_debug_from(config_path, verbose=False): """Run the api's server in dev mode. Note: This is called only once (contrast with the production mode in run_from_webserver function) Args: conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs - 'secret_key' the flask secret key Returns: Never """ load_controllers() conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf host = conf.get('host', '127.0.0.1') port = conf.get('port') debug = conf.get('debug') log_file = os.path.join(conf['log_dir'], 'web-ui.log') logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO, handlers=[logging.FileHandler(log_file), logging.StreamHandler()]) app.run(host=host, port=port, debug=debug) diff --git a/swh/web/ui/renderers.py b/swh/web/ui/renderers.py index 372fd0405..5789bdbd4 100644 --- a/swh/web/ui/renderers.py +++ b/swh/web/ui/renderers.py @@ -1,202 +1,228 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re import yaml import json from docutils.core import publish_parts from docutils.writers.html4css1 import Writer, HTMLTranslator from inspect import cleandoc - +from jinja2 import Markup from flask import request, Response, render_template from flask import g +from pygments import highlight +from pygments.lexers import guess_lexer +from pygments.formatters import HtmlFormatter from swh.web.ui import utils class SWHFilterEnricher(): """Global filter on fields. """ @classmethod def filter_by_fields(cls, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ fields = request.args.get('fields') if fields: fields = set(fields.split(',')) data = utils.filter_field_keys(data, fields) return data class SWHAddLinkHeaderEnricher: """Add link header to response. Mixin intended to be used for example in SWHMultiResponse """ @classmethod def add_link_header(cls, rv, options): """Add Link header in returned value results. Args: rv (dict): with keys: - 'headers': potential headers with 'link-next' and 'link-prev' keys - 'results': containing the result to return Returns: tuple rv, options: If link-headers are present, rv is the returned value present in the 'results' key. Also, options is updated with headers 'Link' containing the 'link-next' and 'link-prev' headers. Otherwise, rv, options stays the same as the input. """ link_headers = [] if 'headers' not in rv: return rv, options rv_headers = rv['headers'] if 'link-next' in rv_headers: link_headers.append('<%s>; rel="next"' % ( rv_headers['link-next'])) if 'link-prev' in rv_headers: link_headers.append('<%s>; rel="previous"' % ( rv_headers['link-prev'])) if link_headers: link_header_str = ','.join(link_headers) headers = options.get('headers', {}) headers.update({ 'Link': link_header_str }) options['headers'] = headers rv.pop('headers') return rv['results'], options return rv, options class SWHMultiResponse(Response, SWHFilterEnricher, SWHAddLinkHeaderEnricher): """ A Flask Response subclass. Override force_type to transform dict/list responses into callable Flask response objects whose mimetype matches the request's Accept header: HTML template render, YAML dump or default to a JSON dump. """ @classmethod def make_response_from_mimetype(cls, rv, options={}): if not (isinstance(rv, list) or isinstance(rv, dict)): return rv def wants_html(best_match): return best_match == 'text/html' and \ request.accept_mimetypes[best_match] > \ request.accept_mimetypes['application/json'] def wants_yaml(best_match): return best_match == 'application/yaml' and \ request.accept_mimetypes[best_match] > \ request.accept_mimetypes['application/json'] rv = cls.filter_by_fields(rv) acc_mime = ['application/json', 'application/yaml', 'text/html'] best_match = request.accept_mimetypes.best_match(acc_mime) rv, options = cls.add_link_header(rv, options) if wants_html(best_match): data = json.dumps(rv, sort_keys=True, indent=4, separators=(',', ': ')) env = g.get('doc_env', {}) env['response_data'] = data env['request'] = request rv = Response(render_template('apidoc.html', **env), content_type='text/html', **options) elif wants_yaml(best_match): rv = Response( yaml.dump(rv), content_type='application/yaml', **options) else: # jsonify is unhappy with lists in Flask 0.10.1, use json.dumps rv = Response( json.dumps(rv), content_type='application/json', **options) return rv @classmethod def force_type(cls, rv, environ=None): if isinstance(rv, dict) or isinstance(rv, list): rv = cls.make_response_from_mimetype(rv) return super().force_type(rv, environ) def error_response(error_code, error): """Private function to create a custom error response. """ error_opts = {'status': error_code} error_data = {'error': str(error)} return SWHMultiResponse.make_response_from_mimetype(error_data, options=error_opts) def urlize_api_links(content): """Utility function for decorating api links in browsable api.""" return re.sub(r'"(/api/.*|/browse/.*)"', r'"\1"', content) class NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] def visit_bullet_list(self, node): self.context.append((self.compact_simple, self.compact_p)) self.compact_p = None self.compact_simple = self.is_compactable(node) self.body.append(self.starttag(node, 'ul', CLASS='docstring')) DOCSTRING_WRITER = Writer() DOCSTRING_WRITER.translator_class = NoHeaderHTMLTranslator def safe_docstring_display(docstring): """ Utility function to htmlize reST-formatted documentation in browsable api. """ docstring = cleandoc(docstring) return publish_parts(docstring, writer=DOCSTRING_WRITER)['html_body'] def revision_id_from_url(url): """Utility function to obtain a revision's ID from its browsing URL.""" return re.sub(r'/browse/revision/([0-9a-f]{40}|[0-9a-f]{64})/.*', r'\1', url) + + +def highlight_source(source_code_as_text): + """Leverage pygments to guess and highlight source code. + + Args + source_code_as_text (str): source code in plain text + + Returns: + Highlighted text if possible or plain text otherwise + + """ + try: + maybe_lexer = guess_lexer(source_code_as_text) + if maybe_lexer: + r = highlight( + source_code_as_text, maybe_lexer, HtmlFormatter(linenos=True)) + else: + r = '
%s
' % source_code_as_text + except: + r = '
%s
' % source_code_as_text + + return Markup(r) diff --git a/swh/web/ui/static/css/pygment.css b/swh/web/ui/static/css/pygment.css new file mode 100644 index 000000000..f76b8b542 --- /dev/null +++ b/swh/web/ui/static/css/pygment.css @@ -0,0 +1,64 @@ +.hll { background-color: #ffffcc } +.c { color: #408080; font-style: italic } /* Comment */ +.err { border: 1px solid #FF0000 } /* Error */ +.k { color: #008000; font-weight: bold } /* Keyword */ +.o { color: #666666 } /* Operator */ +.ch { color: #408080; font-style: italic } /* Comment.Hashbang */ +.cm { color: #408080; font-style: italic } /* Comment.Multiline */ +.cp { color: #BC7A00 } /* Comment.Preproc */ +.cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */ +.c1 { color: #408080; font-style: italic } /* Comment.Single */ +.cs { color: #408080; font-style: italic } /* Comment.Special */ +.gd { color: #A00000 } /* Generic.Deleted */ +.ge { font-style: italic } /* Generic.Emph */ +.gr { color: #FF0000 } /* Generic.Error */ +.gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.gi { color: #00A000 } /* Generic.Inserted */ +.go { color: #888888 } /* Generic.Output */ +.gp { color: #000080; font-weight: bold } /* Generic.Prompt */ +.gs { font-weight: bold } /* Generic.Strong */ +.gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.gt { color: #0044DD } /* Generic.Traceback */ +.kc { color: #008000; font-weight: bold } /* Keyword.Constant */ +.kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ +.kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ +.kp { color: #008000 } /* Keyword.Pseudo */ +.kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ +.kt { color: #B00040 } /* Keyword.Type */ +.m { color: #666666 } /* Literal.Number */ +.s { color: #BA2121 } /* Literal.String */ +.na { color: #7D9029 } /* Name.Attribute */ +.nb { color: #008000 } /* Name.Builtin */ +.nc { color: #0000FF; font-weight: bold } /* Name.Class */ +.no { color: #880000 } /* Name.Constant */ +.nd { color: #AA22FF } /* Name.Decorator */ +.ni { color: #999999; font-weight: bold } /* Name.Entity */ +.ne { color: #D2413A; font-weight: bold } /* Name.Exception */ +.nf { color: #0000FF } /* Name.Function */ +.nl { color: #A0A000 } /* Name.Label */ +.nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ +.nt { color: #008000; font-weight: bold } /* Name.Tag */ +.nv { color: #19177C } /* Name.Variable */ +.ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ +.w { color: #bbbbbb } /* Text.Whitespace */ +.mb { color: #666666 } /* Literal.Number.Bin */ +.mf { color: #666666 } /* Literal.Number.Float */ +.mh { color: #666666 } /* Literal.Number.Hex */ +.mi { color: #666666 } /* Literal.Number.Integer */ +.mo { color: #666666 } /* Literal.Number.Oct */ +.sb { color: #BA2121 } /* Literal.String.Backtick */ +.sc { color: #BA2121 } /* Literal.String.Char */ +.sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ +.s2 { color: #BA2121 } /* Literal.String.Double */ +.se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ +.sh { color: #BA2121 } /* Literal.String.Heredoc */ +.si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ +.sx { color: #008000 } /* Literal.String.Other */ +.sr { color: #BB6688 } /* Literal.String.Regex */ +.s1 { color: #BA2121 } /* Literal.String.Single */ +.ss { color: #19177C } /* Literal.String.Symbol */ +.bp { color: #008000 } /* Name.Builtin.Pseudo */ +.vc { color: #19177C } /* Name.Variable.Class */ +.vg { color: #19177C } /* Name.Variable.Global */ +.vi { color: #19177C } /* Name.Variable.Instance */ +.il { color: #666666 } /* Literal.Number.Integer.Long */ diff --git a/swh/web/ui/templates/content.html b/swh/web/ui/templates/content.html index 7e8ba5537..8935ebb11 100644 --- a/swh/web/ui/templates/content.html +++ b/swh/web/ui/templates/content.html @@ -1,34 +1,34 @@ {% extends "layout.html" %} {% block title %}Content information{% endblock %} {% block content %} {% if message is not none %}

{{ message }}

{% endif %} {% if content is not none %} {% for key in ['sha1', 'sha256', 'sha1_git', 'status', 'length', 'ctime', 'licenses', 'language', 'mimetype', 'encoding'] %}
{{ key }}
{{ content[key] }}
{% endfor %} {% if content['data_url'] is not none %}
data_url
{% endif %} {% if content['data'] is not none %}
data
-
{{ content['data'] }}
+
{{ content['data'] | highlight_source }}
{% endif %} {% if 'decoding_failures' in content %}
(some decoding errors)
{% endif %} {% endif %} {% endblock %} diff --git a/swh/web/ui/templates/layout.html b/swh/web/ui/templates/layout.html index 2327cc583..556763419 100644 --- a/swh/web/ui/templates/layout.html +++ b/swh/web/ui/templates/layout.html @@ -1,61 +1,62 @@ {% block title %}{% endblock %} - The Software Heritage Archive +

{{ self.title() }}

{% with messages = get_flashed_messages(with_categories=true) %} {% if messages %}
{% for category, message in messages %} {% endfor %}
{% endif %} {% endwith %}
{% block content %}{% endblock %}